#-*-coding:utf-8-*-
import scrapy
from CrawlJob.items import CrawljobItem     
import json
from scrapy import Request
from scrapy.contrib.spiders import CrawlSpider
import re
from bs4 import BeautifulSoup
import sys
reload(sys)
sys.setdefaultencoding("utf-8")
class MySpider(CrawlSpider):
    name="job2"
    allowed_domains = ['search.51job.com','jobs.51job.com']#'软件工程师','硬件工程师','数据挖掘','C++','java','计算机','数据分析''测试工程师','需求分析','系统架构','运营','前端','ui','测试工程师','软件开发',
    start_urls = ['http://search.51job.com']
    jobs=['数据挖掘']
    headers = {'User-Agent':'Mozilla/5.0 (Windows; U; Windows NT 6.1; en-US; rv:1.9.1.6) Gecko/20091201 Firefox/3.5.6'}
    def start_requests(self):
        for pro in self.jobs:
            u = "http://search.51job.com/jobsearch/search_result.php?keyword=" + pro.encode('utf-8').decode('utf-8')
            yield scrapy.Request(u, callback=self.parse)
    def parse(self, response):
        for sel in response.css("html body div.dw_wp div#resultList.dw_table div.el p.t1 span a"): 
            url = sel.re('href="(.*?)"')[0]
            yield scrapy.Request(url,callback=self.parse_item)
        nextpage=response.xpath('/html/body/div[2]/div[6]/div/div/div/ul/li[8]/a/@href').extract()[0]
        if nextpage.strip():
            print nextpage
            yield scrapy.Request(nextpage,callback=self.parse)  
    def parse_item(self, response):
        item = CrawljobItem()
        item['minoffer']=self.Getminoffer(response)
        item['maxoffer']=self.Getmaxoffer(response)
        item['city']=self.Getcity(response)
        item['title']=self.Gettitle(response)
        item['company'] =self.Getcompany(response)
        item['location']=self.Getlocation(response)
        item['education']=self.Geteducation(response)
        item['work_experience']=self.Getwork_experience(response)
        item['trades']=self.Gettrades(response)
        item['companyType']=self.GetcompanyType(response)
        item['companySize']=self.GetcompanySize(response)
        item['url']=self.Geturl(response)
        yield item
    # def Getminoffer(self, response):
    #   Minoffer=0.0
    #   offer = response.xpath('/html/body/div[2]/div[2]/div[2]/div/div[1]/strong/text()').extract()[0].encode('utf-8').decode('utf-8')
    #   if offer.strip():
    #       s=re.findall(u'(.*?)-', offer)
    #       if '万/年' in offer:
    #           Minoffer=float(s[0]) * 10000 / 12
    #       elif '万/月' in offer:
    #           Minoffer=float(s[0]) * 10000
    #       else:
    #           Minoffer=float(s[0]) * 1000
    #   else:
    #       Minoffer=10000
    #   return Minoffer
    # def Getmaxoffer(self, response):
    #   Maxoffer=0.0
    #   offer = response.xpath('/html/body/div[2]/div[2]/div[2]/div/div[1]/strong/text()').extract()[0].encode('utf-8').decode('utf-8')
    #   if offer.strip():
    #       if '万/年' in offer:
    #           s = re.findall(u'-(.*?)万', offer)
    #           Maxoffer=float(s[0]) * 10000 / 12
    #       elif '万/月' in offer:
    #           s=re.findall(u'-(.*?)万',offer)
    #           Maxoffer=float(s[0]) * 10000
    #       else:
    #           s=re.findall(u'-(.*?)千',offer)
    #           Maxoffer=float(s[0]) * 1000
    #   else:
    #       Maxoffer=30000
    #   return Maxoffer
    # def Getcity(self, response):
    #   city = response.xpath('//span[@class="lname"]/text()').extract()[0].encode('utf-8').decode('utf-8')
    #   if city=='':
    #       city = '北京'
    #   return city
    # def Geturl(self, response):
    #   url = response.url
    #   return url
    # def Getcompany(self, response):
    #   company=response.xpath('//p[@class="cname"]/a/@title').extract()[0].encode('utf-8').decode('utf-8')
    #   if company=='':
    #       company = "error"
    #   return company
    # def GetcompanyType(self, response):
    #   trades = response.xpath('/html/body/div[2]/div[2]/div[2]/div/div[1]/p[2]/text()').extract()[0].replace(u'\xa0', u'').replace(" ",'').encode('utf-8').decode('utf-8')
    #   trades = ' '.join(trades.split())
    #   if trades == '':
    #       companyType='民营公司'.encode('utf-8').decode('utf-8')
    #       return companyType
    #       pass
    #   companyType = re.split(' |', trades)[0].encode('utf-8').decode('utf-8')
    #   if companyType.strip() == 0:
    #       companyType='民营公司'.encode('utf-8').decode('utf-8')
    #   if '公司' in companyType: 
    #       return companyType
    #   else:
    #       companyType='民营公司'.encode('utf-8').decode('utf-8')
    #       return companyType
    # def GetcompanySize(self, response):
    #   trades = response.xpath('/html/body/div[2]/div[2]/div[2]/div/div[1]/p[2]/text()').extract()[0].replace(u'\xa0', u'').replace(" ",'').encode('utf-8').decode('utf-8')
    #   trades = ' '.join(trades.split())
    #   if trades == '':
    #       companySize='50-150人'.encode('utf-8').decode('utf-8')
    #       return companySize
    #       pass
    #   companySize = re.split(' |', trades)[1].encode('utf-8').decode('utf-8').replace('|', '')
    #   if companySize.strip() == 0:
    #       companySize='50-100人'.encode('utf-8').decode('utf-8')
    #   if '人' in companySize:
    #       return companySize
    #   else:
    #       companySize='50-100人'.encode('utf-8').decode('utf-8')
    #       return companySize
    # def Gettrades(self, response):
    #   trades = response.xpath('/html/body/div[2]/div[2]/div[2]/div/div[1]/p[2]/text()').extract()[0].replace(u'\xa0', u'').replace(" ",'').encode('utf-8').decode('utf-8')
    #   trades = ' '.join(trades.split())
    #   if trades=='':
    #       trades='计算机软件'.encode('utf-8').decode('utf-8')
    #       return trades
    #       pass
    #   for i in xrange(1, 3):
    #       trades = re.split(' |', trades)[i].encode('utf-8').decode('utf-8').replace('|', '')
    #       if '/' in trades:
    #           return trades
    #       else:
    #           trades='计算机软件'.encode('utf-8').decode('utf-8')
    #           return trades
    # def Getlocation(self, response):
    #   location=response.xpath('//p[@class="fp"]/text()').extract()[1].rstrip()
    #   if location=='':
    #       location=response.xpath('//span[@class="lname"]/text()').extract()[0].encode('utf-8').decode('utf-8')
    #       if location=='':
    #           location='北京'
    #           return location
    #       return location
    #   return location
    # def Gettitle(self, response):
    #   title=response.xpath('//div[@class="cn"]/h1/@title').extract()[0].encode('utf-8').decode('utf-8')
    #   return title
    # def Geteducation(self, response):
    #   soup = BeautifulSoup(response.body)
    #   ss = soup.find_all('span',class_='sp4')
    #   education=''
    #   for s in ss:
    #       edu=s.get_text().encode('utf-8').decode('utf-8')
    #       if '专' in edu:
    #           education = edu.encode('utf-8').decode('utf-8')
    #           break
    #       elif '本' in edu:
    #           education = edu.encode('utf-8').decode('utf-8')
    #           break
    #       else:
    #           education='暂无明确要求'.encode('utf-8').decode('utf-8')
    #   return education
    # def Getwork_experience(self, response):
    #   soup = BeautifulSoup(response.body)
    #   ss = soup.find_all('span',class_='sp4')
    #   education=''
    #   for s in ss:
    #       edu=s.get_text().encode('utf-8').decode('utf-8')
    #       if '经验' in edu:
    #           education = edu.encode('utf-8').decode('utf-8')
    #           break
    #       else:
    #           education='暂无要求'.encode('utf-8').decode('utf-8')
    #   return education
    def Getminoffer(self, response):
        Minoffer=0.0
        try:
            offer = response.xpath('/html/body/div[2]/div[2]/div[2]/div/div[1]/strong/text()').extract()[0].encode('utf-8').decode('utf-8')
            if offer.strip():
                s=re.findall(u'(.*?)-', offer)
                if '万/年' in offer:
                    Minoffer=float(s[0]) * 10000 / 12
                elif '万/月' in offer:
                    Minoffer=float(s[0]) * 10000
                else:
                    Minoffer=float(s[0]) * 1000
            else:
                Minoffer=10000
        except:
            Minoffer=10000
        return Minoffer
    def Getmaxoffer(self, response):
        Maxoffer=0.0
        try:
            offer = response.xpath('/html/body/div[2]/div[2]/div[2]/div/div[1]/strong/text()').extract()[0].encode('utf-8').decode('utf-8')
            if offer.strip():
                if '万/年' in offer:
                    s = re.findall(u'-(.*?)万', offer)
                    Maxoffer=float(s[0]) * 10000 / 12
                elif '万/月' in offer:
                    s=re.findall(u'-(.*?)万',offer)
                    Maxoffer=float(s[0]) * 10000
                else:
                    s=re.findall(u'-(.*?)千',offer)
                    Maxoffer=float(s[0]) * 1000
            else:
                Maxoffer=20000
        except:
            Maxoffer=20000
        return Maxoffer
    def Getcity(self, response):
        try:
            city = response.xpath('//span[@class="lname"]/text()').extract()[0].encode('utf-8').decode('utf-8')
            ci=''
            if '-'in city:
                ci=re.findall('(.*?)-',city)
            for c in ci:
                city=''.join(c.encode('utf-8').decode('utf-8'))
            if city=='':
                city = '北京'
        except:
            city='北京'
        return city
    def Geturl(self, response):
        url = response.url
        return url
    def Getcompany(self, response):
        try:
            company=response.xpath('//p[@class="cname"]/a/@title').extract()[0].encode('utf-8').decode('utf-8')
            if company=='':
                company = "None"
        except:
            company='None'
        return company
    def GetcompanyType(self, response):
        try:
            trades = response.xpath('/html/body/div[2]/div[2]/div[2]/div/div[1]/p[2]/text()').extract()[0].replace(u'\xa0', u'').replace(" ",'').encode('utf-8').decode('utf-8')
            trades = ' '.join(trades.split())
            companyType = re.split(' |', trades)[0].encode('utf-8').decode('utf-8')
            if '公司' in companyType: 
                pass
            elif '国企' in companyType:
                companyType='国有企业'.encode('utf-8').decode('utf-8')
            elif '外资' in companyType:
                pass
            else:
                companyType='民营公司'.encode('utf-8').decode('utf-8')
        except:
            companyType='民营公司'
        return companyType
    def GetcompanySize(self, response):
        trades = response.xpath('/html/body/div[2]/div[2]/div[2]/div/div[1]/p[2]/text()').extract()[0].replace(u'\xa0', u'').replace(" ",'').encode('utf-8').decode('utf-8')
        trades = ' '.join(trades.split())
        if trades == '':
            companySize='50-150人'.encode('utf-8').decode('utf-8')
            return companySize
        try:
            companySize = re.split(' |', trades)[1].encode('utf-8').decode('utf-8').replace('|', '')
            if companySize.strip() == 0:
                companySize='50-100人'.encode('utf-8').decode('utf-8')
            if '人' in companySize:
                return companySize
            else:
                companySize='50-100人'.encode('utf-8').decode('utf-8')
                return companySize
        except:
            companySize='50-100人'.encode('utf-8').decode('utf-8')
        return companySize
    def Gettrades(self, response):
        trades = response.xpath('/html/body/div[2]/div[2]/div[2]/div/div[1]/p[2]/text()').extract()[0].replace(u'\xa0', u'').replace(" ",'').encode('utf-8').decode('utf-8')
        trades = ' '.join(trades.split())
        if trades=='':
            trades='计算机软件'.encode('utf-8').decode('utf-8')
            return trades
        # for i in xrange(1, 3):
        try:
            trades = re.split(' |', trades)[2].encode('utf-8').decode('utf-8').replace('|', '')
            if '/' in trades:
                    return trades
            else:
                trades='计算机软件'.encode('utf-8').decode('utf-8')
                return trades
            # trades = ''.join(trades.split())
        except:
            trades='计算机软件'.encode('utf-8').decode('utf-8')
        return trades
    def Getlocation(self, response):
        try:
            location=response.xpath('//p[@class="fp"]/text()').extract()[1].rstrip()
            if location=='':
                location=response.xpath('//span[@class="lname"]/text()').extract()[0].encode('utf-8').decode('utf-8')
                if location=='':
                    location='北京'
        except:
            location='北京'
        return location
    def Gettitle(self, response):
        try:
            title=response.xpath('//div[@class="cn"]/h1/@title').extract()[0].encode('utf-8').decode('utf-8')
        except:
            title="None"
        return title
    def Geteducation(self, response):
        try:
            soup = BeautifulSoup(response.body)
            ss = soup.find_all('span',class_='sp4')
            education=''
            for s in ss:
                edu=s.get_text().encode('utf-8').decode('utf-8')
                if '专' in edu:
                    education = edu.encode('utf-8').decode('utf-8')
                    break
                elif '本' in edu:
                    education = edu.encode('utf-8').decode('utf-8')
                    break
                else:
                    education='暂无明确要求'.encode('utf-8').decode('utf-8')
        except:
            education='暂无明确要求'.encode('utf-8').decode('utf-8')
        return education
    def Getwork_experience(self, response):
        try:
            soup = BeautifulSoup(response.body)
            ss = soup.find_all('span',class_='sp4')
            education=''
            for s in ss:
                edu=s.get_text().encode('utf-8').decode('utf-8')
                if '经验' in edu:
                    education = edu.encode('utf-8').decode('utf-8')
                    break
                else:
                    education='暂无要求'.encode('utf-8').decode('utf-8')
        except:
            education='暂无要求'.encode('utf-8').decode('utf-8')
        return education
